{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "使用jieba拆分中文语句\n",
    "\"\"\"\n",
    "import jieba\n",
    "import jieba.analyse as ja\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入停用词\n",
    "stopword_path='stopwords/cn_stopwords.txt'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#使用正则，去掉常用标调符合以及空格\n",
    "\n",
    "def drop_punctuation(text):\n",
    "    punc = '~`!#$%^&*()_+-=|\\';\":/.,?><~·！@#￥%……&*（）——+-=“：’；、。，？》《{} '\n",
    "    new_text=re.sub(r\"[%s]+\" %punc, \"\",text)\n",
    "    return new_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'距离川沙公路较近但是公交指示不对如果是蔡陆线的话会非常麻烦建议用别的路线房间较为简单'"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "t1='1    距离川沙公路较近,但是公交指示不对,如果是\"蔡陆线\"的话,会非常麻烦.建议用别的路线.房间较为简单.'\n",
    "\n",
    "qt = drop_punctuation(t1)\n",
    "qt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['距离',\n",
       " '川沙',\n",
       " '公路',\n",
       " '较近',\n",
       " '但是',\n",
       " '公交',\n",
       " '指示',\n",
       " '不',\n",
       " '对',\n",
       " '如果',\n",
       " '是',\n",
       " '蔡陆线',\n",
       " '的话',\n",
       " '会',\n",
       " '非常',\n",
       " '麻烦',\n",
       " '建议',\n",
       " '用',\n",
       " '别的',\n",
       " '路线',\n",
       " '房间',\n",
       " '较为简单']"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "#没有去除停用词\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['蔡陆线',\n",
       " '较为简单',\n",
       " '川沙',\n",
       " '较近',\n",
       " '公交',\n",
       " '麻烦',\n",
       " '路线',\n",
       " '指示',\n",
       " '房间',\n",
       " '距离',\n",
       " '公路',\n",
       " '建议',\n",
       " '非常']"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "#使用jieba.analyse.extract_tags去除停用词，好处，快，直接得结果。不好的地方，失去了原始顺序。不建议\n",
    "ja.set_stop_words(stopword_path)\n",
    "cjt=ja.extract_tags(qt)\n",
    "cjt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对比停用词，使用循环方式去除掉了。缺点：当文本量大时，比较占内存。优点：保证顺序\n",
    "stopwords_list=[]\n",
    "with open(stopword_path,'r',encoding='utf8') as wr:\n",
    "    o_list=wr.readlines()\n",
    "    stopwords_list=[i.strip() for i in o_list]\n",
    "#stopwords_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_stopwords(text,stopwords_list):\n",
    "    ct=jieba.lcut(qt)\n",
    "    clear_text=[i for i in ct if i not in stopwords_list]\n",
    "    return clear_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['距离',\n",
       " '川沙',\n",
       " '公路',\n",
       " '较近',\n",
       " '公交',\n",
       " '指示',\n",
       " '蔡陆线',\n",
       " '会',\n",
       " '非常',\n",
       " '麻烦',\n",
       " '建议',\n",
       " '路线',\n",
       " '房间',\n",
       " '较为简单']"
      ]
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "test1=drop_stopwords(qt,stopwords_list)\n",
    "test1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}